





<!DOCTYPE html>
<html class="writer-html5" lang="zh-CN" >
<head>
  <meta charset="utf-8">
  
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  
  <title>Optimizing Operators with Auto-scheduling &mdash; tvm 0.8.dev1982 文档</title>
  

  
  <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/css/bootstrap.min.css" integrity="sha384-Gn5384xqQ1aoWXA+058RXPxPg6fy4IWvTNh0E263XmFcJlSAwiGgFAW/dAiS6JXm" crossorigin="anonymous">
  <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
  <link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
  <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
  <link rel="stylesheet" href="../_static/gallery.css" type="text/css" />
  <link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
  <link rel="stylesheet" href="../_static/css/tlcpack_theme.css" type="text/css" />

  
  
    <link rel="shortcut icon" href="../_static/tvm-logo-square.png"/>
  

  
  
  
  
    
      <script type="text/javascript" id="documentation_options" data-url_root="../" src="../_static/documentation_options.js"></script>
        <script data-url_root="../" id="documentation_options" src="../_static/documentation_options.js"></script>
        <script src="../_static/jquery.js"></script>
        <script src="../_static/underscore.js"></script>
        <script src="../_static/doctools.js"></script>
        <script src="../_static/translations.js"></script>
    
    <script type="text/javascript" src="../_static/js/theme.js"></script>

    
    <script type="text/javascript" src="../_static/js/tlcpack_theme.js"></script>
    <link rel="index" title="索引" href="../genindex.html" />
    <link rel="search" title="搜索" href="../search.html" />
    <link rel="next" title="Cross Compilation and RPC" href="cross_compilation_and_rpc.html" />
    <link rel="prev" title="Optimizing Operators with Schedule Templates and AutoTVM" href="autotvm_matmul_x86.html" /> 
</head>

<body class="wy-body-for-nav">

   
  <div class="wy-grid-for-nav">
    
    
<header class="header">
    <div class="innercontainer">
      <div class="headerInner d-flex justify-content-between align-items-center">
          <div class="headerLogo">
               <a href="https://tvm.apache.org/"><img src=https://tvm.apache.org/assets/images/logo.svg alt="logo"></a>
          </div>

          <div id="headMenu" class="headerNav">
            <button type="button" id="closeHeadMenu" class="navCloseBtn"><img src="../_static/img/close-icon.svg" alt="Close"></button>
             <ul class="nav">
                <li class="nav-item">
                   <a class="nav-link" href=https://tvm.apache.org/community>Community</a>
                </li>
                <li class="nav-item">
                   <a class="nav-link" href=https://tvm.apache.org/download>Download</a>
                </li>
                <li class="nav-item">
                   <a class="nav-link" href=https://tvm.apache.org/vta>VTA</a>
                </li>
                <li class="nav-item">
                   <a class="nav-link" href=https://tvm.apache.org/blog>Blog</a>
                </li>
                <li class="nav-item">
                   <a class="nav-link" href=https://tvm.apache.org/docs>Docs</a>
                </li>
                <li class="nav-item">
                   <a class="nav-link" href=https://tvmconf.org>Conference</a>
                </li>
                <li class="nav-item">
                   <a class="nav-link" href=https://github.com/apache/tvm/>Github</a>
                </li>
                <li class="nav-item">
                   <a class="nav-link" href=https://tvmchinese.github.io/declaration_zh_CN.html>About-Translators</a>
                </li>
             </ul>
               <div class="responsivetlcdropdown">
                 <button type="button" class="btn-link">
                   ASF
                 </button>
                 <ul>
                     <li>
                       <a href=https://apache.org/>Apache Homepage</a>
                     </li>
                     <li>
                       <a href=https://www.apache.org/licenses/>License</a>
                     </li>
                     <li>
                       <a href=https://www.apache.org/foundation/sponsorship.html>Sponsorship</a>
                     </li>
                     <li>
                       <a href=https://www.apache.org/security/>Security</a>
                     </li>
                     <li>
                       <a href=https://www.apache.org/foundation/thanks.html>Thanks</a>
                     </li>
                     <li>
                       <a href=https://www.apache.org/events/current-event>Events</a>
                     </li>
                     <li>
                       <a href=https://www.zhihu.com/column/c_1429578595417563136>Zhihu</a>
                     </li>
                 </ul>
               </div>
          </div>
            <div class="responsiveMenuIcon">
              <button type="button" id="menuBtn" class="btn-menu"><img src="../_static/img/menu-icon.svg" alt="Menu Icon"></button>
            </div>

            <div class="tlcDropdown">
              <div class="dropdown">
                <button type="button" class="btn-link dropdown-toggle" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">
                  ASF
                </button>
                <div class="dropdown-menu dropdown-menu-right">
                  <ul>
                     <li>
                       <a href=https://apache.org/>Apache Homepage</a>
                     </li>
                     <li>
                       <a href=https://www.apache.org/licenses/>License</a>
                     </li>
                     <li>
                       <a href=https://www.apache.org/foundation/sponsorship.html>Sponsorship</a>
                     </li>
                     <li>
                       <a href=https://www.apache.org/security/>Security</a>
                     </li>
                     <li>
                       <a href=https://www.apache.org/foundation/thanks.html>Thanks</a>
                     </li>
                     <li>
                       <a href=https://www.apache.org/events/current-event>Events</a>
                     </li>
                     <li>
                       <a href=https://www.zhihu.com/column/c_1429578595417563136>Zhihu</a>
                     </li>
                  </ul>
                </div>
              </div>
          </div>
       </div>
    </div>
 </header>
 
    <nav data-toggle="wy-nav-shift" class="wy-nav-side fixed">
      <div class="wy-side-scroll">
        <div class="wy-side-nav-search" >
          

          
            <a href="../index.html">
          

          
            
            <img src="../_static/tvm-logo-small.png" class="logo" alt="Logo"/>
          
          </a>

          
            
            
                <div class="version">
                  0.8.dev1982
                </div>
            
          

          
<div role="search">
  <form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
    <input type="text" name="q" placeholder="Search docs" />
    <input type="hidden" name="check_keywords" value="yes" />
    <input type="hidden" name="area" value="default" />
  </form>
</div>

          
        </div>

        
        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
          
            
            
              
            
            
              <p class="caption" role="heading"><span class="caption-text">如何开始</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../install/index.html">安装 TVM</a></li>
<li class="toctree-l1"><a class="reference internal" href="../contribute/index.html">贡献者指南</a></li>
</ul>
<p class="caption" role="heading"><span class="caption-text">用户引导</span></p>
<ul class="current">
<li class="toctree-l1 current"><a class="reference internal" href="index.html">User Tutorial</a><ul class="current">
<li class="toctree-l2"><a class="reference internal" href="introduction.html">介绍</a></li>
<li class="toctree-l2"><a class="reference internal" href="introduction.html#an-overview-of-tvm-and-model-optimization">TVM和模型优化的概述</a></li>
<li class="toctree-l2"><a class="reference internal" href="install.html">安装 TVM</a></li>
<li class="toctree-l2"><a class="reference internal" href="tvmc_command_line_driver.html">使用TVMC编译和优化一个模型</a></li>
<li class="toctree-l2"><a class="reference internal" href="autotvm_relay_x86.html">Compiling and Optimizing a Model with the Python Interface (AutoTVM)</a></li>
<li class="toctree-l2"><a class="reference internal" href="tensor_expr_get_started.html">使用张量表达式来处理运算符</a></li>
<li class="toctree-l2"><a class="reference internal" href="autotvm_matmul_x86.html">Optimizing Operators with Schedule Templates and AutoTVM</a></li>
<li class="toctree-l2 current"><a class="current reference internal" href="#">Optimizing Operators with Auto-scheduling</a><ul>
<li class="toctree-l3"><a class="reference internal" href="#defining-the-matrix-multiplication">Defining the Matrix Multiplication</a></li>
<li class="toctree-l3"><a class="reference internal" href="#create-the-search-task">Create the search task</a></li>
<li class="toctree-l3"><a class="reference internal" href="#set-parameters-for-auto-scheduler">Set Parameters for Auto-Scheduler</a></li>
<li class="toctree-l3"><a class="reference internal" href="#run-the-search">Run the search</a></li>
<li class="toctree-l3"><a class="reference internal" href="#inspecting-the-optimized-schedule">Inspecting the Optimized Schedule</a></li>
<li class="toctree-l3"><a class="reference internal" href="#check-correctness-and-evaluate-performance">Check correctness and evaluate performance</a></li>
<li class="toctree-l3"><a class="reference internal" href="#using-the-record-file">Using the record file</a></li>
<li class="toctree-l3"><a class="reference internal" href="#final-notes-and-summary">最终说明总结</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="cross_compilation_and_rpc.html">Cross Compilation and RPC</a></li>
<li class="toctree-l2"><a class="reference internal" href="relay_quick_start.html">编译深度学习模型的快速开始教程</a></li>
<li class="toctree-l2"><a class="reference internal" href="intro_topi.html">Introduction to TOPI</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../how_to/index.html">How To Guides</a></li>
</ul>
<p class="caption" role="heading"><span class="caption-text">开发者引导</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../dev/tutorial/index.html">Developer Tutorial</a></li>
<li class="toctree-l1"><a class="reference internal" href="../dev/how_to/how_to.html">开发者指南</a></li>
</ul>
<p class="caption" role="heading"><span class="caption-text">架构指南</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../arch/index.html">Design and Architecture</a></li>
</ul>
<p class="caption" role="heading"><span class="caption-text">主题引导</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../topic/microtvm/index.html">microTVM：裸机使用TVM</a></li>
<li class="toctree-l1"><a class="reference internal" href="../topic/vta/index.html">VTA: Versatile Tensor Accelerator</a></li>
</ul>
<p class="caption" role="heading"><span class="caption-text">参考指南</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../reference/langref/index.html">语言参考</a></li>
<li class="toctree-l1"><a class="reference internal" href="../reference/api/python/index.html">Python API</a></li>
<li class="toctree-l1"><a class="reference internal" href="../reference/api/links.html">Other APIs</a></li>
<li class="toctree-l1"><a class="reference internal" href="../reference/publications.html">Publications</a></li>
<li class="toctree-l1"><a class="reference internal" href="../genindex.html">索引</a></li>
</ul>

            
          
        </div>
        
      </div>
    </nav>

    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
      
      <nav class="wy-nav-top" aria-label="top navigation" data-toggle="wy-nav-top">
        
            <div class="togglemenu">

            </div>
            <div class="nav-content">
              <!-- tvm -->
              Table of content
            </div>
        
      </nav>


      <div class="wy-nav-content">
        
        <div class="rst-content">
        

          




















<div role="navigation" aria-label="breadcrumbs navigation">

  <ul class="wy-breadcrumbs">
    
      <li><a href="../index.html">Docs</a> <span class="br-arrow">></span></li>
        
          <li><a href="index.html">User Tutorial</a> <span class="br-arrow">></span></li>
        
      <li>Optimizing Operators with Auto-scheduling</li>
    
    
      <li class="wy-breadcrumbs-aside">
        
            
            <a href="../_sources/tutorial/auto_scheduler_matmul_x86.rst.txt" rel="nofollow"> <img src="../_static//img/source.svg" alt="viewsource"/></a>
          
        
      </li>
    
  </ul>

  
  <hr/>
</div>
          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
           <div itemprop="articleBody">
            
  <div class="sphx-glr-download-link-note admonition note">
<p class="admonition-title">注解</p>
<p>点击 <a class="reference internal" href="#sphx-glr-download-tutorial-auto-scheduler-matmul-x86-py"><span class="std std-ref">此处</span></a> 获取完整示例代码</p>
</div>
<div class="sphx-glr-example-title section" id="optimizing-operators-with-auto-scheduling">
<span id="sphx-glr-tutorial-auto-scheduler-matmul-x86-py"></span><h1>Optimizing Operators with Auto-scheduling<a class="headerlink" href="#optimizing-operators-with-auto-scheduling" title="永久链接至标题">¶</a></h1>
<p><strong>作者</strong>: <a class="reference external" href="https://github.com/merrymercy">Lianmin Zheng</a>,             <a class="reference external" href="https://github.com/jcf94/">Chengfan Jia</a></p>
<p>In this tutorial, we will show how TVM’s Auto Scheduling feature can find
optimal schedules without the need for writing a custom template.</p>
<p>Different from the template-based <a class="reference internal" href="autotvm_matmul_x86.html"><span class="doc">AutoTVM</span></a> which relies on
manual templates to define the search space, the auto-scheduler does not
require any templates.  Users only need to write the computation declaration
without any schedule commands or templates.  The auto-scheduler can
automatically generate a large search space and find a good schedule in the
space.</p>
<p>We use matrix multiplication as an example in this tutorial.</p>
<div class="admonition note">
<p class="admonition-title">注解</p>
<p>Note that this tutorial will not run on Windows or recent versions of macOS. To
get it to run, you will need to wrap the body of this tutorial in a <code class="code docutils literal notranslate"><span class="pre">if</span>
<span class="pre">__name__</span> <span class="pre">==</span> <span class="pre">&quot;__main__&quot;:</span></code> block.</p>
</div>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">os</span>

<span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
<span class="kn">import</span> <span class="nn">tvm</span>
<span class="kn">from</span> <span class="nn">tvm</span> <span class="k">import</span> <span class="n">te</span><span class="p">,</span> <span class="n">auto_scheduler</span>
</pre></div>
</div>
<div class="section" id="defining-the-matrix-multiplication">
<h2>Defining the Matrix Multiplication<a class="headerlink" href="#defining-the-matrix-multiplication" title="永久链接至标题">¶</a></h2>
<p>To start, we define a matrix multiplication with a bias addition.  Note that
this uses standard operations available in TVMs Tensor Expression language.
The major difference is the use of the <cite>auto_sceduler</cite> decorator at the top
of the function definition.  The function should return a list of
input/output tensors.  From these tensors, the auto-scheduler can get the
whole computational graph.</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="nd">@auto_scheduler</span><span class="o">.</span><span class="n">register_workload</span>  <span class="c1"># Note the auto_scheduler decorator</span>
<span class="k">def</span> <span class="nf">matmul_add</span><span class="p">(</span><span class="n">N</span><span class="p">,</span> <span class="n">L</span><span class="p">,</span> <span class="n">M</span><span class="p">,</span> <span class="n">dtype</span><span class="p">):</span>
    <span class="n">A</span> <span class="o">=</span> <span class="n">te</span><span class="o">.</span><span class="n">placeholder</span><span class="p">((</span><span class="n">N</span><span class="p">,</span> <span class="n">L</span><span class="p">),</span> <span class="n">name</span><span class="o">=</span><span class="s2">&quot;A&quot;</span><span class="p">,</span> <span class="n">dtype</span><span class="o">=</span><span class="n">dtype</span><span class="p">)</span>
    <span class="n">B</span> <span class="o">=</span> <span class="n">te</span><span class="o">.</span><span class="n">placeholder</span><span class="p">((</span><span class="n">L</span><span class="p">,</span> <span class="n">M</span><span class="p">),</span> <span class="n">name</span><span class="o">=</span><span class="s2">&quot;B&quot;</span><span class="p">,</span> <span class="n">dtype</span><span class="o">=</span><span class="n">dtype</span><span class="p">)</span>
    <span class="n">C</span> <span class="o">=</span> <span class="n">te</span><span class="o">.</span><span class="n">placeholder</span><span class="p">((</span><span class="n">N</span><span class="p">,</span> <span class="n">M</span><span class="p">),</span> <span class="n">name</span><span class="o">=</span><span class="s2">&quot;C&quot;</span><span class="p">,</span> <span class="n">dtype</span><span class="o">=</span><span class="n">dtype</span><span class="p">)</span>

    <span class="n">k</span> <span class="o">=</span> <span class="n">te</span><span class="o">.</span><span class="n">reduce_axis</span><span class="p">((</span><span class="mi">0</span><span class="p">,</span> <span class="n">L</span><span class="p">),</span> <span class="n">name</span><span class="o">=</span><span class="s2">&quot;k&quot;</span><span class="p">)</span>
    <span class="n">matmul</span> <span class="o">=</span> <span class="n">te</span><span class="o">.</span><span class="n">compute</span><span class="p">(</span>
        <span class="p">(</span><span class="n">N</span><span class="p">,</span> <span class="n">M</span><span class="p">),</span>
        <span class="k">lambda</span> <span class="n">i</span><span class="p">,</span> <span class="n">j</span><span class="p">:</span> <span class="n">te</span><span class="o">.</span><span class="n">sum</span><span class="p">(</span><span class="n">A</span><span class="p">[</span><span class="n">i</span><span class="p">,</span> <span class="n">k</span><span class="p">]</span> <span class="o">*</span> <span class="n">B</span><span class="p">[</span><span class="n">k</span><span class="p">,</span> <span class="n">j</span><span class="p">],</span> <span class="n">axis</span><span class="o">=</span><span class="n">k</span><span class="p">),</span>
        <span class="n">name</span><span class="o">=</span><span class="s2">&quot;matmul&quot;</span><span class="p">,</span>
        <span class="n">attrs</span><span class="o">=</span><span class="p">{</span><span class="s2">&quot;layout_free_placeholders&quot;</span><span class="p">:</span> <span class="p">[</span><span class="n">B</span><span class="p">]},</span>  <span class="c1"># enable automatic layout transform for tensor B</span>
    <span class="p">)</span>
    <span class="n">out</span> <span class="o">=</span> <span class="n">te</span><span class="o">.</span><span class="n">compute</span><span class="p">((</span><span class="n">N</span><span class="p">,</span> <span class="n">M</span><span class="p">),</span> <span class="k">lambda</span> <span class="n">i</span><span class="p">,</span> <span class="n">j</span><span class="p">:</span> <span class="n">matmul</span><span class="p">[</span><span class="n">i</span><span class="p">,</span> <span class="n">j</span><span class="p">]</span> <span class="o">+</span> <span class="n">C</span><span class="p">[</span><span class="n">i</span><span class="p">,</span> <span class="n">j</span><span class="p">],</span> <span class="n">name</span><span class="o">=</span><span class="s2">&quot;out&quot;</span><span class="p">)</span>

    <span class="k">return</span> <span class="p">[</span><span class="n">A</span><span class="p">,</span> <span class="n">B</span><span class="p">,</span> <span class="n">C</span><span class="p">,</span> <span class="n">out</span><span class="p">]</span>
</pre></div>
</div>
</div>
<div class="section" id="create-the-search-task">
<h2>Create the search task<a class="headerlink" href="#create-the-search-task" title="永久链接至标题">¶</a></h2>
<p>With the function defined, we can now create the task for the auto_scheduler
to search against. We specify the particular parameters for this matrix
multiplication, in this case a multiplication of to square matricies of size
1024x1024. We then create a search task with N=L=M=1024 and dtype=”float32”</p>
<div class="admonition note">
<p class="admonition-title">注解</p>
<p>Improve performance with custom targets
In order for TVM to take full advantage of specific hardware platforms,
you will want to manuall specify your CPU capabilities. For example:
- replace “llvm” below with “llvm -mcpu=core-avx2” to enable AVX2
- replace “llvm” below with “llvm -mcpu=skylake-avx512” to enable AVX-512</p>
</div>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">target</span> <span class="o">=</span> <span class="n">tvm</span><span class="o">.</span><span class="n">target</span><span class="o">.</span><span class="n">Target</span><span class="p">(</span><span class="s2">&quot;llvm&quot;</span><span class="p">)</span>
<span class="n">N</span> <span class="o">=</span> <span class="n">L</span> <span class="o">=</span> <span class="n">M</span> <span class="o">=</span> <span class="mi">1024</span>
<span class="n">task</span> <span class="o">=</span> <span class="n">tvm</span><span class="o">.</span><span class="n">auto_scheduler</span><span class="o">.</span><span class="n">SearchTask</span><span class="p">(</span><span class="n">func</span><span class="o">=</span><span class="n">matmul_add</span><span class="p">,</span> <span class="n">args</span><span class="o">=</span><span class="p">(</span><span class="n">N</span><span class="p">,</span> <span class="n">L</span><span class="p">,</span> <span class="n">M</span><span class="p">,</span> <span class="s2">&quot;float32&quot;</span><span class="p">),</span> <span class="n">target</span><span class="o">=</span><span class="n">target</span><span class="p">)</span>

<span class="c1"># Inspect the computational graph</span>
<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Computational DAG:&quot;</span><span class="p">)</span>
<span class="nb">print</span><span class="p">(</span><span class="n">task</span><span class="o">.</span><span class="n">compute_dag</span><span class="p">)</span>
</pre></div>
</div>
<p class="sphx-glr-script-out">输出:</p>
<div class="sphx-glr-script-out highlight-none notranslate"><div class="highlight"><pre><span></span>Computational DAG:
A = PLACEHOLDER [1024, 1024]
B = PLACEHOLDER [1024, 1024]
matmul(i, j) += (A[i, k]*B[k, j])
C = PLACEHOLDER [1024, 1024]
out(i, j) = (matmul[i, j] + C[i, j])
</pre></div>
</div>
</div>
<div class="section" id="set-parameters-for-auto-scheduler">
<h2>Set Parameters for Auto-Scheduler<a class="headerlink" href="#set-parameters-for-auto-scheduler" title="永久链接至标题">¶</a></h2>
<p>Next, we set parameters for the auto-scheduler.</p>
<ul class="simple">
<li><p><code class="code docutils literal notranslate"><span class="pre">num_measure_trials</span></code> is the number of measurement trials we can use
during the search.  We only make 10 trials in this tutorial for a fast
demonstration. In practice, 1000 is a good value for the search to converge.
You can do more trials according to your time budget.</p></li>
<li><p>In addition, we use <code class="code docutils literal notranslate"><span class="pre">RecordToFile</span></code> to log measurement records into a
file <cite>matmul.json</cite>.  The measurement records can be used to query the history
best, resume the search, and do more analyses later.</p></li>
<li><p>see <a class="reference internal" href="../reference/api/python/auto_scheduler.html#tvm.auto_scheduler.TuningOptions" title="tvm.auto_scheduler.TuningOptions"><code class="xref any py py-class docutils literal notranslate"><span class="pre">auto_scheduler.TuningOptions</span></code></a> for more parameters</p></li>
</ul>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">log_file</span> <span class="o">=</span> <span class="s2">&quot;matmul.json&quot;</span>
<span class="n">tune_option</span> <span class="o">=</span> <span class="n">auto_scheduler</span><span class="o">.</span><span class="n">TuningOptions</span><span class="p">(</span>
    <span class="n">num_measure_trials</span><span class="o">=</span><span class="mi">10</span><span class="p">,</span>
    <span class="n">measure_callbacks</span><span class="o">=</span><span class="p">[</span><span class="n">auto_scheduler</span><span class="o">.</span><span class="n">RecordToFile</span><span class="p">(</span><span class="n">log_file</span><span class="p">)],</span>
    <span class="n">verbose</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span>
<span class="p">)</span>
</pre></div>
</div>
</div>
<div class="section" id="run-the-search">
<h2>Run the search<a class="headerlink" href="#run-the-search" title="永久链接至标题">¶</a></h2>
<p>Now we get all inputs ready. Pretty simple, isn’t it?  We can kick off the
search and let the auto-scheduler do its magic.  After some measurement
trials, we can load the best schedule from the log file and apply it.</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="c1"># Run auto-tuning (search)</span>
<span class="n">task</span><span class="o">.</span><span class="n">tune</span><span class="p">(</span><span class="n">tune_option</span><span class="p">)</span>
<span class="c1"># Apply the best schedule</span>
<span class="n">sch</span><span class="p">,</span> <span class="n">args</span> <span class="o">=</span> <span class="n">task</span><span class="o">.</span><span class="n">apply_best</span><span class="p">(</span><span class="n">log_file</span><span class="p">)</span>
</pre></div>
</div>
<p class="sphx-glr-script-out">输出:</p>
<div class="sphx-glr-script-out highlight-none notranslate"><div class="highlight"><pre><span></span>
</pre></div>
</div>
</div>
<div class="section" id="inspecting-the-optimized-schedule">
<h2>Inspecting the Optimized Schedule<a class="headerlink" href="#inspecting-the-optimized-schedule" title="永久链接至标题">¶</a></h2>
<p>We can lower the schedule to see the IR after auto-scheduling.  The
auto-scheduler correctly performs optimizations including multi-level tiling,
layout transformation, parallelization, vectorization, unrolling, and
operator fusion.</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Lowered TIR:&quot;</span><span class="p">)</span>
<span class="nb">print</span><span class="p">(</span><span class="n">tvm</span><span class="o">.</span><span class="n">lower</span><span class="p">(</span><span class="n">sch</span><span class="p">,</span> <span class="n">args</span><span class="p">,</span> <span class="n">simple_mode</span><span class="o">=</span><span class="kc">True</span><span class="p">))</span>
</pre></div>
</div>
<p class="sphx-glr-script-out">输出:</p>
<div class="sphx-glr-script-out highlight-none notranslate"><div class="highlight"><pre><span></span>Lowered TIR:
primfn(A_1: handle, B_1: handle, C_1: handle, out_1: handle) -&gt; ()
  attr = {&quot;from_legacy_te_schedule&quot;: True, &quot;global_symbol&quot;: &quot;main&quot;, &quot;tir.noalias&quot;: True}
  buffers = {C: Buffer(C_2: Pointer(float32), float32, [1024, 1024], []),
             out: Buffer(out_2: Pointer(float32), float32, [1024, 1024], []),
             A: Buffer(A_2: Pointer(float32), float32, [1024, 1024], []),
             B: Buffer(B_2: Pointer(float32), float32, [1024, 1024], [])}
  buffer_map = {A_1: A, B_1: B, C_1: C, out_1: out} {
  allocate(auto_scheduler_layout_transform: Pointer(global float32), float32, [1048576]), storage_scope = global {
    for (ax0.ax1.fused.ax2.fused: int32, 0, 16) &quot;parallel&quot; {
      for (ax3: int32, 0, 64) {
        for (ax4: int32, 0, 8) {
          for (ax5: int32, 0, 16) {
            for (ax6: int32, 0, 8) {
              auto_scheduler_layout_transform[(((((ax0.ax1.fused.ax2.fused*65536) + (ax3*1024)) + (ax4*128)) + (ax5*8)) + ax6)] = (float32*)B_2[(((((ax3*16384) + (ax5*1024)) + (ax0.ax1.fused.ax2.fused*64)) + (ax4*8)) + ax6)]
            }
          }
        }
      }
    }
    for (i.outer.j.outer.fused: int32, 0, 64) &quot;parallel&quot; {
      allocate(matmul: Pointer(global float32), float32, [16384]), storage_scope = global {
        for (i.outer.outer.inner: int32, 0, 4) {
          for (j.outer.outer.inner: int32, 0, 4) {
            for (i.outer.inner.init: int32, 0, 8) {
              matmul[ramp((((i.outer.outer.inner*4096) + (i.outer.inner.init*512)) + (j.outer.outer.inner*64)), 1, 8)] = broadcast(0f32, 8)
              matmul[ramp(((((i.outer.outer.inner*4096) + (i.outer.inner.init*512)) + (j.outer.outer.inner*64)) + 256), 1, 8)] = broadcast(0f32, 8)
              matmul[ramp(((((i.outer.outer.inner*4096) + (i.outer.inner.init*512)) + (j.outer.outer.inner*64)) + 8), 1, 8)] = broadcast(0f32, 8)
              matmul[ramp(((((i.outer.outer.inner*4096) + (i.outer.inner.init*512)) + (j.outer.outer.inner*64)) + 264), 1, 8)] = broadcast(0f32, 8)
              matmul[ramp(((((i.outer.outer.inner*4096) + (i.outer.inner.init*512)) + (j.outer.outer.inner*64)) + 16), 1, 8)] = broadcast(0f32, 8)
              matmul[ramp(((((i.outer.outer.inner*4096) + (i.outer.inner.init*512)) + (j.outer.outer.inner*64)) + 272), 1, 8)] = broadcast(0f32, 8)
              matmul[ramp(((((i.outer.outer.inner*4096) + (i.outer.inner.init*512)) + (j.outer.outer.inner*64)) + 24), 1, 8)] = broadcast(0f32, 8)
              matmul[ramp(((((i.outer.outer.inner*4096) + (i.outer.inner.init*512)) + (j.outer.outer.inner*64)) + 280), 1, 8)] = broadcast(0f32, 8)
              matmul[ramp(((((i.outer.outer.inner*4096) + (i.outer.inner.init*512)) + (j.outer.outer.inner*64)) + 32), 1, 8)] = broadcast(0f32, 8)
              matmul[ramp(((((i.outer.outer.inner*4096) + (i.outer.inner.init*512)) + (j.outer.outer.inner*64)) + 288), 1, 8)] = broadcast(0f32, 8)
              matmul[ramp(((((i.outer.outer.inner*4096) + (i.outer.inner.init*512)) + (j.outer.outer.inner*64)) + 40), 1, 8)] = broadcast(0f32, 8)
              matmul[ramp(((((i.outer.outer.inner*4096) + (i.outer.inner.init*512)) + (j.outer.outer.inner*64)) + 296), 1, 8)] = broadcast(0f32, 8)
              matmul[ramp(((((i.outer.outer.inner*4096) + (i.outer.inner.init*512)) + (j.outer.outer.inner*64)) + 48), 1, 8)] = broadcast(0f32, 8)
              matmul[ramp(((((i.outer.outer.inner*4096) + (i.outer.inner.init*512)) + (j.outer.outer.inner*64)) + 304), 1, 8)] = broadcast(0f32, 8)
              matmul[ramp(((((i.outer.outer.inner*4096) + (i.outer.inner.init*512)) + (j.outer.outer.inner*64)) + 56), 1, 8)] = broadcast(0f32, 8)
              matmul[ramp(((((i.outer.outer.inner*4096) + (i.outer.inner.init*512)) + (j.outer.outer.inner*64)) + 312), 1, 8)] = broadcast(0f32, 8)
            }
            for (k.outer: int32, 0, 64) {
              for (i.outer.inner: int32, 0, 8) {
                for (j.outer.inner: int32, 0, 8) {
                  for (k.inner: int32, 0, 16) {
                    matmul[ramp(((((i.outer.outer.inner*4096) + (i.outer.inner*512)) + (j.outer.outer.inner*64)) + (j.outer.inner*8)), 1, 8)] = ((float32x8*)matmul[ramp(((((i.outer.outer.inner*4096) + (i.outer.inner*512)) + (j.outer.outer.inner*64)) + (j.outer.inner*8)), 1, 8)] + (broadcast((float32*)A_2[(((((floordiv(i.outer.j.outer.fused, 4)*65536) + (i.outer.outer.inner*16384)) + (i.outer.inner*2048)) + (k.outer*16)) + k.inner)], 8)*(float32x8*)auto_scheduler_layout_transform[ramp((((((floormod(i.outer.j.outer.fused, 4)*262144) + (j.outer.outer.inner*65536)) + (k.outer*1024)) + (j.outer.inner*128)) + (k.inner*8)), 1, 8)]))
                    matmul[ramp((((((i.outer.outer.inner*4096) + (i.outer.inner*512)) + (j.outer.outer.inner*64)) + (j.outer.inner*8)) + 256), 1, 8)] = ((float32x8*)matmul[ramp((((((i.outer.outer.inner*4096) + (i.outer.inner*512)) + (j.outer.outer.inner*64)) + (j.outer.inner*8)) + 256), 1, 8)] + (broadcast((float32*)A_2[((((((floordiv(i.outer.j.outer.fused, 4)*65536) + (i.outer.outer.inner*16384)) + (i.outer.inner*2048)) + (k.outer*16)) + k.inner) + 1024)], 8)*(float32x8*)auto_scheduler_layout_transform[ramp((((((floormod(i.outer.j.outer.fused, 4)*262144) + (j.outer.outer.inner*65536)) + (k.outer*1024)) + (j.outer.inner*128)) + (k.inner*8)), 1, 8)]))
                  }
                }
              }
            }
          }
        }
        for (i.inner: int32, 0, 64) {
          for (j.inner: int32, 0, 256) {
            out_2[((((floordiv(i.outer.j.outer.fused, 4)*65536) + (i.inner*1024)) + (floormod(i.outer.j.outer.fused, 4)*256)) + j.inner)] = ((float32*)matmul[((i.inner*256) + j.inner)] + (float32*)C_2[((((floordiv(i.outer.j.outer.fused, 4)*65536) + (i.inner*1024)) + (floormod(i.outer.j.outer.fused, 4)*256)) + j.inner)])
          }
        }
      }
    }
  }
}
</pre></div>
</div>
</div>
<div class="section" id="check-correctness-and-evaluate-performance">
<h2>Check correctness and evaluate performance<a class="headerlink" href="#check-correctness-and-evaluate-performance" title="永久链接至标题">¶</a></h2>
<p>We build the binary and check its correctness and performance.</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">func</span> <span class="o">=</span> <span class="n">tvm</span><span class="o">.</span><span class="n">build</span><span class="p">(</span><span class="n">sch</span><span class="p">,</span> <span class="n">args</span><span class="p">,</span> <span class="n">target</span><span class="p">)</span>
<span class="n">a_np</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">uniform</span><span class="p">(</span><span class="n">size</span><span class="o">=</span><span class="p">(</span><span class="n">N</span><span class="p">,</span> <span class="n">L</span><span class="p">))</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">float32</span><span class="p">)</span>
<span class="n">b_np</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">uniform</span><span class="p">(</span><span class="n">size</span><span class="o">=</span><span class="p">(</span><span class="n">L</span><span class="p">,</span> <span class="n">M</span><span class="p">))</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">float32</span><span class="p">)</span>
<span class="n">c_np</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">uniform</span><span class="p">(</span><span class="n">size</span><span class="o">=</span><span class="p">(</span><span class="n">N</span><span class="p">,</span> <span class="n">M</span><span class="p">))</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">float32</span><span class="p">)</span>
<span class="n">out_np</span> <span class="o">=</span> <span class="n">a_np</span><span class="o">.</span><span class="n">dot</span><span class="p">(</span><span class="n">b_np</span><span class="p">)</span> <span class="o">+</span> <span class="n">c_np</span>

<span class="n">dev</span> <span class="o">=</span> <span class="n">tvm</span><span class="o">.</span><span class="n">cpu</span><span class="p">()</span>
<span class="n">a_tvm</span> <span class="o">=</span> <span class="n">tvm</span><span class="o">.</span><span class="n">nd</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="n">a_np</span><span class="p">,</span> <span class="n">device</span><span class="o">=</span><span class="n">dev</span><span class="p">)</span>
<span class="n">b_tvm</span> <span class="o">=</span> <span class="n">tvm</span><span class="o">.</span><span class="n">nd</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="n">b_np</span><span class="p">,</span> <span class="n">device</span><span class="o">=</span><span class="n">dev</span><span class="p">)</span>
<span class="n">c_tvm</span> <span class="o">=</span> <span class="n">tvm</span><span class="o">.</span><span class="n">nd</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="n">c_np</span><span class="p">,</span> <span class="n">device</span><span class="o">=</span><span class="n">dev</span><span class="p">)</span>
<span class="n">out_tvm</span> <span class="o">=</span> <span class="n">tvm</span><span class="o">.</span><span class="n">nd</span><span class="o">.</span><span class="n">empty</span><span class="p">(</span><span class="n">out_np</span><span class="o">.</span><span class="n">shape</span><span class="p">,</span> <span class="n">device</span><span class="o">=</span><span class="n">dev</span><span class="p">)</span>
<span class="n">func</span><span class="p">(</span><span class="n">a_tvm</span><span class="p">,</span> <span class="n">b_tvm</span><span class="p">,</span> <span class="n">c_tvm</span><span class="p">,</span> <span class="n">out_tvm</span><span class="p">)</span>

<span class="c1"># Check results</span>
<span class="n">np</span><span class="o">.</span><span class="n">testing</span><span class="o">.</span><span class="n">assert_allclose</span><span class="p">(</span><span class="n">out_np</span><span class="p">,</span> <span class="n">out_tvm</span><span class="o">.</span><span class="n">numpy</span><span class="p">(),</span> <span class="n">rtol</span><span class="o">=</span><span class="mf">1e-3</span><span class="p">)</span>

<span class="c1"># Evaluate execution time.</span>
<span class="n">evaluator</span> <span class="o">=</span> <span class="n">func</span><span class="o">.</span><span class="n">time_evaluator</span><span class="p">(</span><span class="n">func</span><span class="o">.</span><span class="n">entry_name</span><span class="p">,</span> <span class="n">dev</span><span class="p">,</span> <span class="n">min_repeat_ms</span><span class="o">=</span><span class="mi">500</span><span class="p">)</span>
<span class="nb">print</span><span class="p">(</span>
    <span class="s2">&quot;Execution time of this operator: </span><span class="si">%.3f</span><span class="s2"> ms&quot;</span>
    <span class="o">%</span> <span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">median</span><span class="p">(</span><span class="n">evaluator</span><span class="p">(</span><span class="n">a_tvm</span><span class="p">,</span> <span class="n">b_tvm</span><span class="p">,</span> <span class="n">c_tvm</span><span class="p">,</span> <span class="n">out_tvm</span><span class="p">)</span><span class="o">.</span><span class="n">results</span><span class="p">)</span> <span class="o">*</span> <span class="mi">1000</span><span class="p">)</span>
<span class="p">)</span>
</pre></div>
</div>
<p class="sphx-glr-script-out">输出:</p>
<div class="sphx-glr-script-out highlight-none notranslate"><div class="highlight"><pre><span></span>Execution time of this operator: 35.062 ms
</pre></div>
</div>
</div>
<div class="section" id="using-the-record-file">
<h2>Using the record file<a class="headerlink" href="#using-the-record-file" title="永久链接至标题">¶</a></h2>
<p>During the search, all measurement records are logged into the record file
“matmul.json”. The measurement records can be used to re-apply search
results, resume the search, and perform other analyses.</p>
<p>Here is an example where we load the best schedule from a file, and print the
equivalent python schedule API. This can be used for debugging and learning
the behavior of the auto-scheduler.</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Equivalent python schedule:&quot;</span><span class="p">)</span>
<span class="nb">print</span><span class="p">(</span><span class="n">task</span><span class="o">.</span><span class="n">print_best</span><span class="p">(</span><span class="n">log_file</span><span class="p">))</span>
</pre></div>
</div>
<p class="sphx-glr-script-out">输出:</p>
<div class="sphx-glr-script-out highlight-none notranslate"><div class="highlight"><pre><span></span>Equivalent python schedule:
matmul_i, matmul_j, matmul_k = tuple(matmul.op.axis) + tuple(matmul.op.reduce_axis)
out_i, out_j = tuple(out.op.axis) + tuple(out.op.reduce_axis)
matmul_i_o_i, matmul_i_i = s[matmul].split(matmul_i, factor=2)
matmul_i_o_o_i, matmul_i_o_i = s[matmul].split(matmul_i_o_i, factor=8)
matmul_i_o_o_o, matmul_i_o_o_i = s[matmul].split(matmul_i_o_o_i, factor=4)
matmul_j_o_i, matmul_j_i = s[matmul].split(matmul_j, factor=8)
matmul_j_o_o_i, matmul_j_o_i = s[matmul].split(matmul_j_o_i, factor=8)
matmul_j_o_o_o, matmul_j_o_o_i = s[matmul].split(matmul_j_o_o_i, factor=4)
matmul_k_o, matmul_k_i = s[matmul].split(matmul_k, factor=16)
s[matmul].reorder(matmul_i_o_o_o, matmul_j_o_o_o, matmul_i_o_o_i, matmul_j_o_o_i, matmul_k_o, matmul_i_o_i, matmul_j_o_i, matmul_k_i, matmul_i_i, matmul_j_i)
out_i_o, out_i_i = s[out].split(out_i, factor=64)
out_j_o, out_j_i = s[out].split(out_j, factor=256)
s[out].reorder(out_i_o, out_j_o, out_i_i, out_j_i)
s[matmul].compute_at(s[out], out_j_o)
out_i_o_j_o_fused = s[out].fuse(out_i_o, out_j_o)
s[out].parallel(out_i_o_j_o_fused)
s[matmul].pragma(matmul_i_o_o_o, &quot;auto_unroll_max_step&quot;, 16)
s[matmul].pragma(matmul_i_o_o_o, &quot;unroll_explicit&quot;, True)
s[matmul].vectorize(matmul_j_i)
</pre></div>
</div>
<p>A more complicated example is to resume the search.  In this case, we need to
create the search policy and cost model by ourselves and resume the status of
search policy and cost model with the log file.  In the example below we
resume the status and do more 5 trials.</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">resume_search</span><span class="p">(</span><span class="n">task</span><span class="p">,</span> <span class="n">log_file</span><span class="p">):</span>
    <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Resume search:&quot;</span><span class="p">)</span>
    <span class="n">cost_model</span> <span class="o">=</span> <span class="n">auto_scheduler</span><span class="o">.</span><span class="n">XGBModel</span><span class="p">()</span>
    <span class="n">cost_model</span><span class="o">.</span><span class="n">update_from_file</span><span class="p">(</span><span class="n">log_file</span><span class="p">)</span>
    <span class="n">search_policy</span> <span class="o">=</span> <span class="n">auto_scheduler</span><span class="o">.</span><span class="n">SketchPolicy</span><span class="p">(</span>
        <span class="n">task</span><span class="p">,</span> <span class="n">cost_model</span><span class="p">,</span> <span class="n">init_search_callbacks</span><span class="o">=</span><span class="p">[</span><span class="n">auto_scheduler</span><span class="o">.</span><span class="n">PreloadMeasuredStates</span><span class="p">(</span><span class="n">log_file</span><span class="p">)]</span>
    <span class="p">)</span>
    <span class="n">tune_option</span> <span class="o">=</span> <span class="n">auto_scheduler</span><span class="o">.</span><span class="n">TuningOptions</span><span class="p">(</span>
        <span class="n">num_measure_trials</span><span class="o">=</span><span class="mi">5</span><span class="p">,</span> <span class="n">measure_callbacks</span><span class="o">=</span><span class="p">[</span><span class="n">auto_scheduler</span><span class="o">.</span><span class="n">RecordToFile</span><span class="p">(</span><span class="n">log_file</span><span class="p">)]</span>
    <span class="p">)</span>
    <span class="n">task</span><span class="o">.</span><span class="n">tune</span><span class="p">(</span><span class="n">tune_option</span><span class="p">,</span> <span class="n">search_policy</span><span class="o">=</span><span class="n">search_policy</span><span class="p">)</span>


<span class="n">resume_search</span><span class="p">(</span><span class="n">task</span><span class="p">,</span> <span class="n">log_file</span><span class="p">)</span>
</pre></div>
</div>
<p class="sphx-glr-script-out">输出:</p>
<div class="sphx-glr-script-out highlight-none notranslate"><div class="highlight"><pre><span></span>Resume search:
/usr/local/lib/python3.6/dist-packages/xgboost/training.py:17: UserWarning: Old style callback is deprecated.  See: https://xgboost.readthedocs.io/en/latest/python/callbacks.html
  warnings.warn(f&#39;Old style callback is deprecated.  See: {link}&#39;, UserWarning)
</pre></div>
</div>
</div>
<div class="section" id="final-notes-and-summary">
<h2>最终说明总结<a class="headerlink" href="#final-notes-and-summary" title="永久链接至标题">¶</a></h2>
<p>In this tutorial, we have shown how to use the TVM Auto-Scheduler to
automatically optimize a matrix multiplication, without the need to specify a
search template.  It ends a series of examples that starts from the Tensor
Expression (TE) language that demonstrates how TVM can optimize computational
operations.</p>
<div class="sphx-glr-footer class sphx-glr-footer-example docutils container" id="sphx-glr-download-tutorial-auto-scheduler-matmul-x86-py">
<div class="sphx-glr-download docutils container">
<p><a class="reference download internal" download="" href="../_downloads/eac4389b114db015e95cb3cdf8b86b83/auto_scheduler_matmul_x86.py"><code class="xref download docutils literal notranslate"><span class="pre">Python</span> <span class="pre">源码下载:</span> <span class="pre">auto_scheduler_matmul_x86.py</span></code></a></p>
</div>
<div class="sphx-glr-download docutils container">
<p><a class="reference download internal" download="" href="../_downloads/246d4b8509474fd9046e69f6cc9b7f87/auto_scheduler_matmul_x86.ipynb"><code class="xref download docutils literal notranslate"><span class="pre">Jupyter</span> <span class="pre">notebook</span> <span class="pre">下载:</span> <span class="pre">auto_scheduler_matmul_x86.ipynb</span></code></a></p>
</div>
</div>
<p class="sphx-glr-signature"><a class="reference external" href="https://sphinx-gallery.github.io">Gallery generated by Sphinx-Gallery</a></p>
</div>
</div>


           </div>
           
          </div>
          

<footer>

    <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
      
        <a href="cross_compilation_and_rpc.html" class="btn btn-neutral float-right" title="Cross Compilation and RPC" accesskey="n" rel="next">下一个 <span class="fa fa-arrow-circle-right"></span></a>
      
      
        <a href="autotvm_matmul_x86.html" class="btn btn-neutral float-left" title="Optimizing Operators with Schedule Templates and AutoTVM" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left"></span> 上一个</a>
      
    </div>

<div id="button" class="backtop"><img src="../_static//img/right.svg" alt="backtop"/> </div>
<section class="footerSec">
    <div class="footerHeader">
      <ul class="d-flex align-md-items-center justify-content-between flex-column flex-md-row">
        <li class="copywrite d-flex align-items-center">
          <h5 id="copy-right-info">© 2020 Apache Software Foundation | All right reserved</h5>
        </li>
      </ul>

    </div>

    <ul>
      <li class="footernote">Copyright © 2020 The Apache Software Foundation. Apache TVM, Apache, the Apache feather, and the Apache TVM project logo are either trademarks or registered trademarks of the Apache Software Foundation.</li>
    </ul>

</section>
</footer>
        </div>
      </div>

    </section>

  </div>
  

    <script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.12.9/umd/popper.min.js" integrity="sha384-ApNbgh9B+Y1QKtv3Rn7W3mgPxhU9K/ScQsAP7hUibX39j7fakFPskvXusvfa0b4Q" crossorigin="anonymous"></script>
    <script src="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/js/bootstrap.min.js" integrity="sha384-JZR6Spejh4U02d8jOt6vLEHfe/JQGiRRSQQxSfFWpi1MquVdAyjUar5+76PVCmYl" crossorigin="anonymous"></script>

  </body>
  <script type="text/javascript">
      jQuery(function () {
          SphinxRtdTheme.Navigation.enable(true);
      });
  </script>

  
  
    
    <!-- Theme Analytics -->
    <script>
    (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
      (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
      m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
    })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');

    ga('create', 'UA-75982049-2', 'auto');
    ga('send', 'pageview');
    </script>

    
   

</body>
</html>